{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "VYbRe_-wY8bU"
   },
   "source": [
    "安装依赖"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "kyRwnMNrNMQJ",
    "outputId": "a5383399-e804-4a57-bb6d-e6d23cee4a39"
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Looking in indexes: http://mirrors.aliyun.com/pypi/simple/\n",
      "Requirement already satisfied: datasets in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (3.3.2)\n",
      "Collecting emoji\n",
      "  Downloading http://mirrors.aliyun.com/pypi/packages/91/db/a0335710caaa6d0aebdaa65ad4df789c15d89b7babd9a30277838a7d9aac/emoji-2.14.1-py3-none-any.whl (590 kB)\n",
      "\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m590.6/590.6 kB\u001b[0m \u001b[31m30.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n",
      "\u001b[?25hRequirement already satisfied: scikit-learn in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (1.0.2)\n",
      "Requirement already satisfied: filelock in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (3.13.1)\n",
      "Requirement already satisfied: numpy>=1.17 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (1.23.5)\n",
      "Requirement already satisfied: pyarrow>=15.0.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (19.0.1)\n",
      "Requirement already satisfied: dill<0.3.9,>=0.3.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (0.3.8)\n",
      "Requirement already satisfied: pandas in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (2.1.2)\n",
      "Requirement already satisfied: requests>=2.32.2 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (2.32.3)\n",
      "Requirement already satisfied: tqdm>=4.66.3 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (4.67.1)\n",
      "Requirement already satisfied: xxhash in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (3.5.0)\n",
      "Requirement already satisfied: multiprocess<0.70.17 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (0.70.16)\n",
      "Requirement already satisfied: fsspec<=2024.12.0,>=2023.1.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from fsspec[http]<=2024.12.0,>=2023.1.0->datasets) (2023.10.0)\n",
      "Requirement already satisfied: aiohttp in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (3.11.12)\n",
      "Requirement already satisfied: huggingface-hub>=0.24.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (0.29.1)\n",
      "Requirement already satisfied: packaging in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (23.2)\n",
      "Requirement already satisfied: pyyaml>=5.1 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from datasets) (6.0.1)\n",
      "Requirement already satisfied: scipy>=1.1.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from scikit-learn) (1.11.3)\n",
      "Requirement already satisfied: joblib>=0.11 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from scikit-learn) (1.3.2)\n",
      "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from scikit-learn) (3.2.0)\n",
      "Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (2.4.6)\n",
      "Requirement already satisfied: aiosignal>=1.1.2 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (1.3.2)\n",
      "Requirement already satisfied: async-timeout<6.0,>=4.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (5.0.1)\n",
      "Requirement already satisfied: attrs>=17.3.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (23.1.0)\n",
      "Requirement already satisfied: frozenlist>=1.1.1 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (1.5.0)\n",
      "Requirement already satisfied: multidict<7.0,>=4.5 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (6.1.0)\n",
      "Requirement already satisfied: propcache>=0.2.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (0.3.0)\n",
      "Requirement already satisfied: yarl<2.0,>=1.17.0 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from aiohttp->datasets) (1.18.3)\n",
      "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from huggingface-hub>=0.24.0->datasets) (4.8.0)\n",
      "Requirement already satisfied: charset-normalizer<4,>=2 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from requests>=2.32.2->datasets) (3.3.2)\n",
      "Requirement already satisfied: idna<4,>=2.5 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from requests>=2.32.2->datasets) (3.4)\n",
      "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from requests>=2.32.2->datasets) (2.0.7)\n",
      "Requirement already satisfied: certifi>=2017.4.17 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from requests>=2.32.2->datasets) (2023.7.22)\n",
      "Requirement already satisfied: python-dateutil>=2.8.2 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from pandas->datasets) (2.8.2)\n",
      "Requirement already satisfied: pytz>=2020.1 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from pandas->datasets) (2023.3.post1)\n",
      "Requirement already satisfied: tzdata>=2022.1 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from pandas->datasets) (2023.3)\n",
      "Requirement already satisfied: six>=1.5 in /home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages (from python-dateutil>=2.8.2->pandas->datasets) (1.16.0)\n",
      "\u001b[33mDEPRECATION: moxing-framework 2.1.16.2ae09d45 has a non-standard version number. pip 24.0 will enforce this behaviour change. A possible replacement is to upgrade to a newer version of moxing-framework or contact the author to suggest that they release a version with a conforming version number. Discussion can be found at https://github.com/pypa/pip/issues/12063\u001b[0m\u001b[33m\n",
      "\u001b[0mInstalling collected packages: emoji\n",
      "Successfully installed emoji-2.14.1\n",
      "\u001b[33mWARNING: Running pip as the 'root' user can result in broken permissions and conflicting behaviour with the system package manager. It is recommended to use a virtual environment instead: https://pip.pypa.io/warnings/venv\u001b[0m\u001b[33m\n",
      "\u001b[0m"
     ]
    }
   ],
   "source": [
    "# !pip install mindspore==2.3.1 mindnlp==0.4.1\n",
    "!pip install datasets emoji scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mindspore\n",
    "mindspore.set_context(device_target='Ascend', device_id=0, pynative_synchronize=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "ZkR0IJfZZDcu"
   },
   "source": [
    "加载数据集"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "IVDaehs4DaBx",
    "outputId": "c1b322b8-42d6-4293-eb93-478ce3b086aa"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n",
      "Generating train split: 5679 examples [00:00, 145319.43 examples/s]\n",
      "Generating test split: 1000 examples [00:00, 72245.83 examples/s]\n",
      "Filter: 100%|██████████| 5679/5679 [00:00<00:00, 122458.75 examples/s]\n",
      "Filter: 100%|██████████| 1000/1000 [00:00<00:00, 68059.52 examples/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "train dataset num_rows:  5678\n",
      "test dataset num_rows:  1000\n",
      "                                               tweet  label\n",
      "0  krazy i dont always get drunk and pass out but...      0\n",
      "1  white kids favorite activities calling people ...      1\n",
      "2  maam did you clear that tweet with the   caref...      0\n",
      "3  wth is that playing missy  i mean seriously rt...      0\n",
      "4           he promised to stand with the muzzies so      0\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "\n",
    "dataset = load_dataset(\"/tmp/code/hate_speech_twitter\")\n",
    "# dataset = load_dataset(\"thefrankhsu/hate_speech_twitter\")\n",
    "train_dataset = dataset['train'].remove_columns('categories').filter(lambda x: x['tweet'] is not None and x['label'] in [0, 1])\n",
    "test_dataset = dataset['test'].remove_columns('categories').filter(lambda x: x['tweet'] is not None and x['label'] in [0, 1])\n",
    "\n",
    "print(\"train dataset num_rows: \", train_dataset.num_rows)\n",
    "print(\"test dataset num_rows: \", test_dataset.num_rows)\n",
    "print(train_dataset.with_format(\"pandas\")[:5])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "jtl4kIsVZGjT"
   },
   "source": [
    "加载模型"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "qmopJYcgNo7t",
    "outputId": "50ea9f1a-e606-4600-eb10-c48a2fc03669"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Dumping model to file cache /tmp/jieba.cache\n",
      "Loading model cost 1.412 seconds.\n",
      "Prefix dict has been built successfully.\n",
      "/home/ma-user/anaconda3/envs/MindSpore/lib/python3.9/site-packages/mindnlp/transformers/tokenization_utils_base.py:1526: FutureWarning: `clean_up_tokenization_spaces` was not set. It will be set to `True` by default. This behavior will be depracted, and will be then set to `False` by default. \n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[MS_ALLOC_CONF]Runtime config:  enable_vmm:True  vmm_align_size:2MB\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Some weights of RobertaForSequenceClassification were not initialized from the model checkpoint at /tmp/code/bertweet-base and are newly initialized: ['classifier.dense.bias', 'classifier.dense.weight', 'classifier.out_proj.bias', 'classifier.out_proj.weight']\n",
      "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
     ]
    }
   ],
   "source": [
    "from mindnlp.transformers import BertweetTokenizer, AutoModelForSequenceClassification\n",
    "\n",
    "tokenizer = BertweetTokenizer.from_pretrained(\"/tmp/code/bertweet-base\")\n",
    "# tokenizer = BertweetTokenizer.from_pretrained(\"vinai/bertweet-base\")\n",
    "model = AutoModelForSequenceClassification.from_pretrained(\"/tmp/code/bertweet-base\", num_labels=2)\n",
    "# model = AutoModelForSequenceClassification.from_pretrained(\"vinai/bertweet-base\", num_labels=2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "B_dSEjcGZIew"
   },
   "source": [
    "数据预处理"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "id": "b7Gd_yl4RD1z"
   },
   "outputs": [],
   "source": [
    "import mindspore\n",
    "from mindspore.dataset import GeneratorDataset, transforms\n",
    "\n",
    "class HSTDataset:\n",
    "    def __init__(self, dataset):\n",
    "        self.dataset = dataset\n",
    "\n",
    "    def __len__(self):\n",
    "        return len(self.dataset)\n",
    "\n",
    "    def __getitem__(self, idx):\n",
    "        if not isinstance(idx, int):\n",
    "            idx = int(idx)\n",
    "        return self.dataset[idx]['tweet'], self.dataset[idx]['label']\n",
    "\n",
    "def process_dataset(source, tokenizer, max_seq_len=64, batch_size=32, shuffle=True, drop_remainder=False):\n",
    "    is_ascend = mindspore.get_context('device_target') == 'Ascend'\n",
    "\n",
    "    column_names = [\"tweet\", \"label\"]\n",
    "\n",
    "    dataset = GeneratorDataset(source, column_names=column_names, shuffle=shuffle)\n",
    "    # transforms\n",
    "    type_cast_op = transforms.TypeCast(mindspore.int32)\n",
    "    def tokenize_and_pad(text):\n",
    "        if is_ascend:\n",
    "            tokenized = tokenizer(text, padding='max_length', truncation=True, max_length=max_seq_len)\n",
    "        else:\n",
    "            tokenized = tokenizer(text)\n",
    "        return tokenized['input_ids'], tokenized['attention_mask']\n",
    "    # map dataset\n",
    "    dataset = dataset.map(operations=tokenize_and_pad, input_columns=\"tweet\", output_columns=['input_ids', 'attention_mask'])\n",
    "    dataset = dataset.map(operations=[type_cast_op], input_columns=\"label\", output_columns='labels')\n",
    "    # # batch dataset\n",
    "    if is_ascend:\n",
    "        dataset = dataset.batch(batch_size, drop_remainder=drop_remainder)\n",
    "    else:\n",
    "        dataset = dataset.padded_batch(batch_size, drop_remainder=drop_remainder, \n",
    "                                       pad_info={'input_ids': (None, tokenizer.pad_token_id),\n",
    "                                                'attention_mask': (None, 0)})\n",
    "\n",
    "    return dataset\n",
    "\n",
    "train_dataset = process_dataset(HSTDataset(train_dataset), tokenizer, drop_remainder=True)\n",
    "test_dataset = process_dataset(HSTDataset(test_dataset), tokenizer, shuffle=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'input_ids': Tensor(shape=[32, 64], dtype=Int64, value=\n",
      "[[    0,    37,  1484 ...     1,     1,     1],\n",
      " [    0,   462, 10898 ...     1,     1,     1],\n",
      " [    0,   111,   112 ...     1,     1,     1],\n",
      " ...\n",
      " [    0,     6,    83 ...     1,     1,     1],\n",
      " [    0,   322,  1472 ...     1,     1,     1],\n",
      " [    0,   460, 51959 ...     1,     1,     1]]), 'attention_mask': Tensor(shape=[32, 64], dtype=Int64, value=\n",
      "[[1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " ...\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0],\n",
      " [1, 1, 1 ... 0, 0, 0]]), 'labels': Tensor(shape=[32], dtype=Int32, value= [1, 0, 0, 0, 0, 1, 0, 1, 0, 1, 0, 1, 0, 0, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, \n",
      " 0, 0, 0, 1, 0, 0, 0, 0])}\n"
     ]
    }
   ],
   "source": [
    "print(next(train_dataset.create_dict_iterator()))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "tVJXYysimlz6"
   },
   "source": [
    "评估函数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "id": "pnhao-Krmoak"
   },
   "outputs": [],
   "source": [
    "from sklearn.metrics import accuracy_score, precision_recall_fscore_support\n",
    "import numpy as np\n",
    "\n",
    "def compute_metrics(eval_pred):\n",
    "    logits, labels = eval_pred\n",
    "    predictions = np.argmax(logits, axis=-1)\n",
    "\n",
    "    accuracy = accuracy_score(labels, predictions)\n",
    "    precision, recall, f1, _ = precision_recall_fscore_support(labels, predictions, average='binary')\n",
    "\n",
    "    return {\n",
    "        'accuracy': accuracy,\n",
    "        'precision': precision,\n",
    "        'recall': recall,\n",
    "        'f1': f1\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "yVQO2666jtlb"
   },
   "source": [
    "设置训练参数"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "id": "j8mB_fnkjxHK"
   },
   "outputs": [],
   "source": [
    "from mindnlp.engine import TrainingArguments\n",
    "\n",
    "training_args = TrainingArguments(\n",
    "    output_dir=\"./bertweet_finetune/trainer_output\",\n",
    "    evaluation_strategy=\"epoch\",\n",
    "    save_strategy=\"epoch\",\n",
    "    save_total_limit=2,\n",
    "    load_best_model_at_end=True,\n",
    "    metric_for_best_model='accuracy',\n",
    "    greater_is_better=True,\n",
    "    per_device_train_batch_size=32,\n",
    "    per_device_eval_batch_size=32,\n",
    "    num_train_epochs=5,\n",
    "    learning_rate=2e-5,\n",
    "    weight_decay=0.01,\n",
    "    logging_strategy=\"epoch\"\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "dMG2LKnzkeYg"
   },
   "source": [
    "加载Trainer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/"
    },
    "id": "GxKv25xlkdq7",
    "outputId": "80aaf393-55c4-45bd-bd9d-900492065969"
   },
   "outputs": [],
   "source": [
    "from mindnlp.engine import Trainer\n",
    "\n",
    "trainer = Trainer(\n",
    "    model=model,\n",
    "    args=training_args,\n",
    "    train_dataset=train_dataset,\n",
    "    eval_dataset=test_dataset,\n",
    "    tokenizer=tokenizer,\n",
    "    compute_metrics=compute_metrics\n",
    ")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "id": "O9W1BPf_ocwe"
   },
   "source": [
    "训练"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {
    "colab": {
     "base_uri": "https://localhost:8080/",
     "height": 609
    },
    "id": "29XLe3Q1oeg7",
    "outputId": "158f7c37-b20c-422a-b5cf-b3db545bfdf3"
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  0%|          | 0/885 [00:00<?, ?it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "|\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 177/885 [03:14<10:29,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "/\r"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██        | 177/885 [03:15<10:29,  1.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.3052, 'learning_rate': 1.6000000000000003e-05, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 2/32 [00:00<00:08,  3.59it/s]\u001b[A\n",
      "  9%|▉         | 3/32 [00:00<00:06,  4.67it/s]\u001b[A\n",
      " 12%|█▎        | 4/32 [00:00<00:05,  5.56it/s]\u001b[A\n",
      " 16%|█▌        | 5/32 [00:00<00:04,  6.25it/s]\u001b[A\n",
      " 19%|█▉        | 6/32 [00:01<00:03,  6.79it/s]\u001b[A\n",
      " 22%|██▏       | 7/32 [00:01<00:03,  7.17it/s]\u001b[A\n",
      " 25%|██▌       | 8/32 [00:01<00:03,  7.45it/s]\u001b[A\n",
      " 28%|██▊       | 9/32 [00:01<00:03,  7.60it/s]\u001b[A\n",
      " 31%|███▏      | 10/32 [00:01<00:02,  7.71it/s]\u001b[A\n",
      " 34%|███▍      | 11/32 [00:01<00:02,  7.78it/s]\u001b[A\n",
      " 38%|███▊      | 12/32 [00:01<00:02,  7.84it/s]\u001b[A\n",
      " 41%|████      | 13/32 [00:01<00:02,  7.88it/s]\u001b[A\n",
      " 44%|████▍     | 14/32 [00:02<00:02,  7.89it/s]\u001b[A\n",
      " 47%|████▋     | 15/32 [00:02<00:02,  7.92it/s]\u001b[A\n",
      " 50%|█████     | 16/32 [00:02<00:02,  7.98it/s]\u001b[A\n",
      " 53%|█████▎    | 17/32 [00:02<00:01,  8.00it/s]\u001b[A\n",
      " 56%|█████▋    | 18/32 [00:02<00:01,  7.97it/s]\u001b[A\n",
      " 59%|█████▉    | 19/32 [00:02<00:01,  7.97it/s]\u001b[A\n",
      " 62%|██████▎   | 20/32 [00:02<00:01,  7.98it/s]\u001b[A\n",
      " 66%|██████▌   | 21/32 [00:02<00:01,  8.01it/s]\u001b[A\n",
      " 69%|██████▉   | 22/32 [00:03<00:01,  7.99it/s]\u001b[A\n",
      " 72%|███████▏  | 23/32 [00:03<00:01,  7.99it/s]\u001b[A\n",
      " 75%|███████▌  | 24/32 [00:03<00:01,  7.98it/s]\u001b[A\n",
      " 78%|███████▊  | 25/32 [00:03<00:00,  7.99it/s]\u001b[A\n",
      " 81%|████████▏ | 26/32 [00:03<00:00,  7.97it/s]\u001b[A\n",
      " 84%|████████▍ | 27/32 [00:03<00:00,  8.00it/s]\u001b[A\n",
      " 88%|████████▊ | 28/32 [00:03<00:00,  8.01it/s]\u001b[A\n",
      " 91%|█████████ | 29/32 [00:03<00:00,  7.98it/s]\u001b[A\n",
      " 94%|█████████▍| 30/32 [00:04<00:00,  7.96it/s]\u001b[A\n",
      " 97%|█████████▋| 31/32 [00:04<00:00,  7.96it/s]\u001b[A\n",
      "                                                 A\n",
      " 20%|██        | 177/885 [03:21<10:29,  1.12it/s]\n",
      "100%|██████████| 32/32 [00:04<00:00,  7.74it/s]\u001b[A\n",
      "                                               \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8967171907424927, 'eval_accuracy': 0.67, 'eval_precision': 0.9427083333333334, 'eval_recall': 0.362, 'eval_f1': 0.523121387283237, 'eval_runtime': 6.2447, 'eval_samples_per_second': 5.124, 'eval_steps_per_second': 0.16, 'epoch': 1.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|████      | 354/885 [06:26<07:55,  1.12it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.143, 'learning_rate': 1.2e-05, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 2/32 [00:00<00:05,  5.53it/s]\u001b[A\n",
      "  9%|▉         | 3/32 [00:00<00:07,  3.88it/s]\u001b[A\n",
      " 12%|█▎        | 4/32 [00:01<00:08,  3.50it/s]\u001b[A\n",
      " 16%|█▌        | 5/32 [00:01<00:07,  3.72it/s]\u001b[A\n",
      " 19%|█▉        | 6/32 [00:01<00:05,  4.46it/s]\u001b[A\n",
      " 22%|██▏       | 7/32 [00:01<00:04,  5.14it/s]\u001b[A\n",
      " 25%|██▌       | 8/32 [00:01<00:04,  5.71it/s]\u001b[A\n",
      " 28%|██▊       | 9/32 [00:01<00:03,  6.20it/s]\u001b[A\n",
      " 31%|███▏      | 10/32 [00:01<00:03,  6.59it/s]\u001b[A\n",
      " 34%|███▍      | 11/32 [00:02<00:03,  6.95it/s]\u001b[A\n",
      " 38%|███▊      | 12/32 [00:02<00:02,  6.94it/s]\u001b[A\n",
      " 41%|████      | 13/32 [00:02<00:02,  7.18it/s]\u001b[A\n",
      " 44%|████▍     | 14/32 [00:02<00:02,  7.42it/s]\u001b[A\n",
      " 47%|████▋     | 15/32 [00:02<00:02,  7.61it/s]\u001b[A\n",
      " 50%|█████     | 16/32 [00:02<00:02,  7.51it/s]\u001b[A\n",
      " 53%|█████▎    | 17/32 [00:02<00:02,  7.44it/s]\u001b[A\n",
      " 56%|█████▋    | 18/32 [00:03<00:01,  7.48it/s]\u001b[A\n",
      " 59%|█████▉    | 19/32 [00:03<00:01,  7.53it/s]\u001b[A\n",
      " 62%|██████▎   | 20/32 [00:03<00:01,  7.63it/s]\u001b[A\n",
      " 66%|██████▌   | 21/32 [00:03<00:01,  7.86it/s]\u001b[A\n",
      " 69%|██████▉   | 22/32 [00:03<00:01,  7.97it/s]\u001b[A\n",
      " 72%|███████▏  | 23/32 [00:03<00:01,  8.05it/s]\u001b[A\n",
      " 75%|███████▌  | 24/32 [00:03<00:00,  8.15it/s]\u001b[A\n",
      " 78%|███████▊  | 25/32 [00:03<00:00,  8.23it/s]\u001b[A\n",
      " 81%|████████▏ | 26/32 [00:03<00:00,  8.28it/s]\u001b[A\n",
      " 84%|████████▍ | 27/32 [00:04<00:00,  8.32it/s]\u001b[A\n",
      " 88%|████████▊ | 28/32 [00:04<00:00,  8.37it/s]\u001b[A\n",
      " 91%|█████████ | 29/32 [00:04<00:00,  8.40it/s]\u001b[A\n",
      " 94%|█████████▍| 30/32 [00:04<00:00,  8.42it/s]\u001b[A\n",
      " 97%|█████████▋| 31/32 [00:04<00:00,  8.43it/s]\u001b[A\n",
      "                                                 A\n",
      " 40%|████      | 354/885 [06:31<07:55,  1.12it/s]\n",
      "100%|██████████| 32/32 [00:04<00:00,  8.66it/s]\u001b[A\n",
      "                                               \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.8762021064758301, 'eval_accuracy': 0.738, 'eval_precision': 0.9407407407407408, 'eval_recall': 0.508, 'eval_f1': 0.6597402597402598, 'eval_runtime': 5.5101, 'eval_samples_per_second': 5.808, 'eval_steps_per_second': 0.181, 'epoch': 2.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|██████    | 531/885 [09:36<05:27,  1.08it/s]  "
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0963, 'learning_rate': 8.000000000000001e-06, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 2/32 [00:00<00:05,  5.69it/s]\u001b[A\n",
      "  9%|▉         | 3/32 [00:00<00:07,  3.84it/s]\u001b[A\n",
      " 12%|█▎        | 4/32 [00:01<00:08,  3.43it/s]\u001b[A\n",
      " 16%|█▌        | 5/32 [00:01<00:08,  3.20it/s]\u001b[A\n",
      " 19%|█▉        | 6/32 [00:01<00:06,  3.95it/s]\u001b[A\n",
      " 22%|██▏       | 7/32 [00:01<00:05,  4.67it/s]\u001b[A\n",
      " 25%|██▌       | 8/32 [00:01<00:04,  5.33it/s]\u001b[A\n",
      " 28%|██▊       | 9/32 [00:01<00:03,  5.88it/s]\u001b[A\n",
      " 31%|███▏      | 10/32 [00:02<00:03,  6.34it/s]\u001b[A\n",
      " 34%|███▍      | 11/32 [00:02<00:03,  6.74it/s]\u001b[A\n",
      " 38%|███▊      | 12/32 [00:02<00:02,  7.06it/s]\u001b[A\n",
      " 41%|████      | 13/32 [00:02<00:02,  7.27it/s]\u001b[A\n",
      " 44%|████▍     | 14/32 [00:02<00:02,  7.43it/s]\u001b[A\n",
      " 47%|████▋     | 15/32 [00:02<00:02,  7.47it/s]\u001b[A\n",
      " 50%|█████     | 16/32 [00:02<00:02,  7.48it/s]\u001b[A\n",
      " 53%|█████▎    | 17/32 [00:02<00:01,  7.51it/s]\u001b[A\n",
      " 56%|█████▋    | 18/32 [00:03<00:01,  7.49it/s]\u001b[A\n",
      " 59%|█████▉    | 19/32 [00:03<00:01,  7.54it/s]\u001b[A\n",
      " 62%|██████▎   | 20/32 [00:03<00:01,  7.62it/s]\u001b[A\n",
      " 66%|██████▌   | 21/32 [00:03<00:01,  7.65it/s]\u001b[A\n",
      " 69%|██████▉   | 22/32 [00:03<00:01,  7.62it/s]\u001b[A\n",
      " 72%|███████▏  | 23/32 [00:03<00:01,  7.62it/s]\u001b[A\n",
      " 75%|███████▌  | 24/32 [00:03<00:01,  7.61it/s]\u001b[A\n",
      " 78%|███████▊  | 25/32 [00:04<00:00,  7.61it/s]\u001b[A\n",
      " 81%|████████▏ | 26/32 [00:04<00:00,  7.62it/s]\u001b[A\n",
      " 84%|████████▍ | 27/32 [00:04<00:00,  7.60it/s]\u001b[A\n",
      " 88%|████████▊ | 28/32 [00:04<00:00,  7.56it/s]\u001b[A\n",
      " 91%|█████████ | 29/32 [00:04<00:00,  7.49it/s]\u001b[A\n",
      " 94%|█████████▍| 30/32 [00:04<00:00,  7.50it/s]\u001b[A\n",
      " 97%|█████████▋| 31/32 [00:04<00:00,  7.47it/s]\u001b[A\n",
      "                                                 A\n",
      " 60%|██████    | 531/885 [09:41<05:27,  1.08it/s]\n",
      "100%|██████████| 32/32 [00:04<00:00,  7.61it/s]\u001b[A\n",
      "                                               \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.6897300481796265, 'eval_accuracy': 0.79, 'eval_precision': 0.9475308641975309, 'eval_recall': 0.614, 'eval_f1': 0.7451456310679612, 'eval_runtime': 5.406, 'eval_samples_per_second': 5.919, 'eval_steps_per_second': 0.185, 'epoch': 3.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████  | 708/885 [12:44<02:43,  1.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0635, 'learning_rate': 4.000000000000001e-06, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 2/32 [00:00<00:05,  5.44it/s]\u001b[A\n",
      "  9%|▉         | 3/32 [00:00<00:07,  3.92it/s]\u001b[A\n",
      " 12%|█▎        | 4/32 [00:01<00:08,  3.32it/s]\u001b[A\n",
      " 16%|█▌        | 5/32 [00:01<00:09,  2.99it/s]\u001b[A\n",
      " 19%|█▉        | 6/32 [00:01<00:07,  3.56it/s]\u001b[A\n",
      " 22%|██▏       | 7/32 [00:01<00:05,  4.23it/s]\u001b[A\n",
      " 25%|██▌       | 8/32 [00:01<00:05,  4.75it/s]\u001b[A\n",
      " 28%|██▊       | 9/32 [00:02<00:04,  5.42it/s]\u001b[A\n",
      " 31%|███▏      | 10/32 [00:02<00:03,  6.02it/s]\u001b[A\n",
      " 34%|███▍      | 11/32 [00:02<00:03,  6.54it/s]\u001b[A\n",
      " 38%|███▊      | 12/32 [00:02<00:02,  6.88it/s]\u001b[A\n",
      " 41%|████      | 13/32 [00:02<00:02,  7.19it/s]\u001b[A\n",
      " 44%|████▍     | 14/32 [00:02<00:02,  7.45it/s]\u001b[A\n",
      " 47%|████▋     | 15/32 [00:02<00:02,  7.54it/s]\u001b[A\n",
      " 50%|█████     | 16/32 [00:02<00:02,  7.70it/s]\u001b[A\n",
      " 53%|█████▎    | 17/32 [00:03<00:01,  7.82it/s]\u001b[A\n",
      " 56%|█████▋    | 18/32 [00:03<00:01,  7.71it/s]\u001b[A\n",
      " 59%|█████▉    | 19/32 [00:03<00:01,  7.85it/s]\u001b[A\n",
      " 62%|██████▎   | 20/32 [00:03<00:01,  7.96it/s]\u001b[A\n",
      " 66%|██████▌   | 21/32 [00:03<00:01,  8.04it/s]\u001b[A\n",
      " 69%|██████▉   | 22/32 [00:03<00:01,  7.89it/s]\u001b[A\n",
      " 72%|███████▏  | 23/32 [00:03<00:01,  7.98it/s]\u001b[A\n",
      " 75%|███████▌  | 24/32 [00:03<00:00,  8.06it/s]\u001b[A\n",
      " 78%|███████▊  | 25/32 [00:04<00:00,  8.07it/s]\u001b[A\n",
      " 81%|████████▏ | 26/32 [00:04<00:00,  8.13it/s]\u001b[A\n",
      " 84%|████████▍ | 27/32 [00:04<00:00,  8.20it/s]\u001b[A\n",
      " 88%|████████▊ | 28/32 [00:04<00:00,  8.26it/s]\u001b[A\n",
      " 91%|█████████ | 29/32 [00:04<00:00,  8.28it/s]\u001b[A\n",
      " 94%|█████████▍| 30/32 [00:04<00:00,  8.30it/s]\u001b[A\n",
      " 97%|█████████▋| 31/32 [00:04<00:00,  8.33it/s]\u001b[A\n",
      "                                                 A\n",
      " 80%|████████  | 708/885 [12:50<02:43,  1.08it/s]\n",
      "100%|██████████| 32/32 [00:04<00:00,  8.54it/s]\u001b[A\n",
      "                                               \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.7547961473464966, 'eval_accuracy': 0.801, 'eval_precision': 0.943952802359882, 'eval_recall': 0.64, 'eval_f1': 0.7628128724672228, 'eval_runtime': 5.4116, 'eval_samples_per_second': 5.913, 'eval_steps_per_second': 0.185, 'epoch': 4.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 885/885 [15:52<00:00,  1.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'loss': 0.0528, 'learning_rate': 0.0, 'epoch': 5.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n",
      "  0%|          | 0/32 [00:00<?, ?it/s]\u001b[A\n",
      "  6%|▋         | 2/32 [00:00<00:05,  5.07it/s]\u001b[A\n",
      "  9%|▉         | 3/32 [00:00<00:08,  3.36it/s]\u001b[A\n",
      " 12%|█▎        | 4/32 [00:01<00:11,  2.40it/s]\u001b[A\n",
      " 16%|█▌        | 5/32 [00:01<00:10,  2.46it/s]\u001b[A\n",
      " 19%|█▉        | 6/32 [00:01<00:08,  3.19it/s]\u001b[A\n",
      " 22%|██▏       | 7/32 [00:02<00:06,  3.94it/s]\u001b[A\n",
      " 25%|██▌       | 8/32 [00:02<00:05,  4.64it/s]\u001b[A\n",
      " 28%|██▊       | 9/32 [00:02<00:04,  5.31it/s]\u001b[A\n",
      " 31%|███▏      | 10/32 [00:02<00:03,  5.79it/s]\u001b[A\n",
      " 34%|███▍      | 11/32 [00:02<00:03,  6.33it/s]\u001b[A\n",
      " 38%|███▊      | 12/32 [00:02<00:02,  6.70it/s]\u001b[A\n",
      " 41%|████      | 13/32 [00:02<00:02,  6.96it/s]\u001b[A\n",
      " 44%|████▍     | 14/32 [00:03<00:02,  7.15it/s]\u001b[A\n",
      " 47%|████▋     | 15/32 [00:03<00:02,  7.17it/s]\u001b[A\n",
      " 50%|█████     | 16/32 [00:03<00:02,  7.31it/s]\u001b[A\n",
      " 53%|█████▎    | 17/32 [00:03<00:02,  7.27it/s]\u001b[A\n",
      " 56%|█████▋    | 18/32 [00:03<00:01,  7.37it/s]\u001b[A\n",
      " 59%|█████▉    | 19/32 [00:03<00:01,  7.38it/s]\u001b[A\n",
      " 62%|██████▎   | 20/32 [00:03<00:01,  7.38it/s]\u001b[A\n",
      " 66%|██████▌   | 21/32 [00:03<00:01,  7.39it/s]\u001b[A\n",
      " 69%|██████▉   | 22/32 [00:04<00:01,  7.49it/s]\u001b[A\n",
      " 72%|███████▏  | 23/32 [00:04<00:01,  7.42it/s]\u001b[A\n",
      " 75%|███████▌  | 24/32 [00:04<00:01,  7.47it/s]\u001b[A\n",
      " 78%|███████▊  | 25/32 [00:04<00:00,  7.48it/s]\u001b[A\n",
      " 81%|████████▏ | 26/32 [00:04<00:00,  7.50it/s]\u001b[A\n",
      " 84%|████████▍ | 27/32 [00:04<00:00,  7.41it/s]\u001b[A\n",
      " 88%|████████▊ | 28/32 [00:04<00:00,  7.46it/s]\u001b[A\n",
      " 91%|█████████ | 29/32 [00:05<00:00,  7.53it/s]\u001b[A\n",
      " 94%|█████████▍| 30/32 [00:05<00:00,  7.49it/s]\u001b[A\n",
      " 97%|█████████▋| 31/32 [00:05<00:00,  7.46it/s]\u001b[A\n",
      "                                                 A\n",
      "100%|██████████| 885/885 [15:58<00:00,  1.13it/s]\n",
      "100%|██████████| 32/32 [00:05<00:00,  7.50it/s]\u001b[A\n",
      "                                               \u001b[A"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'eval_loss': 0.9358894228935242, 'eval_accuracy': 0.77, 'eval_precision': 0.944078947368421, 'eval_recall': 0.574, 'eval_f1': 0.7139303482587064, 'eval_runtime': 5.9235, 'eval_samples_per_second': 5.402, 'eval_steps_per_second': 0.169, 'epoch': 5.0}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████| 885/885 [16:17<00:00,  1.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'train_runtime': 977.8286, 'train_samples_per_second': 28.962, 'train_steps_per_second': 0.905, 'train_loss': 0.13215566516596045, 'epoch': 5.0}\n",
      "Train over!\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "# Train the model\n",
    "trainer.train()\n",
    "\n",
    "print(\"Train over!\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "inputs: 'on my way to fuck your bitch in the name of The Lord', predict: 'Hate Speech'\n"
     ]
    }
   ],
   "source": [
    "from mindspore import Tensor\n",
    "def predict(text, label=None):\n",
    "    label_map = {0: \"Non Hate Speech\", 1: \"Hate Speech\"}\n",
    "\n",
    "    text_tokenized = Tensor([tokenizer(text).input_ids])\n",
    "    logits = model(text_tokenized)\n",
    "    predict_label = logits[0].asnumpy().argmax()\n",
    "    info = f\"inputs: '{text}', predict: '{label_map[predict_label]}'\"\n",
    "    if label is not None:\n",
    "        info += f\" , label: '{label_map[label]}'\"\n",
    "    print(info)\n",
    "predict(\"on my way to fuck your bitch in the name of The Lord\")"
   ]
  }
 ],
 "metadata": {
  "accelerator": "GPU",
  "colab": {
   "gpuType": "T4",
   "provenance": [],
   "toc_visible": true
  },
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.18"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
